import os
from bs4 import BeautifulSoup #
from selenium import webdriver #浏览器
import time #时间操作
import json

#爬取dolin 商品信息
from selenium.webdriver.common.keys import Keys

from com.py.test.reptleDolinGoodMsg.CommentPage import CommentPage
from com.py.test.reptleDolinGoodMsg.DbOperate import DbOperate
from com.py.test.reptleDolinGoodMsg.MyThread import MyThread


class FunHeadlines:
    def __init__(self):
        self.url = "https://hnbang.com/view/28961.html"
    def start(self):
        print("——开始自动化爬取———")
        #浏览器设置
        options = webdriver.ChromeOptions()
        #设置中文
        options.add_argument('lang=zh_CN.UTF-8')
        #更换头部
        ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
        options.add_argument('user-agent="%s"' % ua)
        driver = webdriver.Chrome(chrome_options=options)
        driver.get(self.url)
        assert "东菱" in driver.title
        #去没用标签
        # [a.extract() for a in html.find_all('div',class_='p-commit')]
        # [a.extract() for a in html.find_all('div',class_='p-shop')]
        # [a.extract() for a in html.find_all('div',class_='p-icons')]
        # [a.extract() for a in html.find_all('div',class_='p-operate')]
        #翻页
        goodsUrls = []
        # while True:
        #     driver.execute_script("$('.pn-next').click()")
        time.sleep(2) #等待
        self.scroll_win(driver,1)  #1 次数
        html = BeautifulSoup(driver.page_source,'lxml')
        goods_list = html.find('div',id="J_goodsList")
        goods_list= goods_list.find_all('div',class_="p-img")
        #获取商品详情页地址
        for pimg in goods_list:
            url = pimg.a['href'].replace('https://','//').replace('//','https://')
            goodsUrls.append(url);
            # if(html.find('a',class_="pn-next disabled")):
            #     break;
        print(goodsUrls)
        driver.quit()
        #访问商品详情页 /开多线程
        # for url in goodsUrls:
        #     CommentPage().start(url)
        c = CommentPage()
        dbo = DbOperate()
        size = len(goodsUrls)
        # step = 100 #100一组
        step = 10 #100一组
        groups = [goodsUrls[i:i+step] for i in range(0,size,step)]
        thread_num = len(groups)  #线程数
        for i in range(thread_num):
            # if(i%4==0):
            # elif:
            print(groups[i])
            t = MyThread(self.runThread,args=(c,dbo,groups[i]))
            t.start()
    def runThread(self,c,dbo,list):
        for item in list:
            c.start(dbo,item)
    # 下拉加载
    def scroll_win(self,driver,times):
        for i in range(times):
            print("开始执行第", str(i + 1),"次下拉操作")
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")  #执行JavaScript实现网页下拉倒底部
            print("第", str(i + 1), "次下拉操作执行完毕")
            time.sleep(3)  # 等待3秒，页面加载出来再执行下拉操作
FunHeadlines().start()